In [2]:
# Set up sys.path so that 'src/spindle_dev' is importable as 'spindle_dev'
import sys
import importlib
from pathlib import Path
project_root = '/data/sarkar_lab/Projects/spindle_dev'
src_path = Path(project_root) / 'src'
if str(src_path) not in sys.path:
sys.path.insert(0, str(src_path))
import spindle_dev
# Reload to pick up code changes without restarting the kernel
importlib.reload(spindle_dev)
Out[2]:
<module 'spindle_dev' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/__init__.py'>
In [3]:
import scanpy as sc
In [4]:
import glob
In [5]:
h5ad_files = glob.glob("/data/sarkar_lab/insitupy_demo_data_xenium/*.h5ad")
In [6]:
h5ad_files
Out[6]:
['/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_skin_melanoma.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_brain_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_kidney_nondiseased.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lung_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_lymph_node_5k.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_pancreatic_cancer.h5ad', '/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_breast_cancer.h5ad']
In [9]:
adata = sc.read_h5ad('/data/sarkar_lab/insitupy_demo_data_xenium/xenium_human_brain_cancer.h5ad')
In [10]:
from pathlib import Path
import spindle_dev
import spindle_dev.metrics as metrics
import spindle_dev.index as index
import spindle_dev.preprocessing as preprocessing
import spindle_dev.plotting as plotting
import spindle_dev.test as test
import spindle_dev.search as search
import spindle_dev.typing as typing
# Reload to pick up code changes without restarting the kernel
import time
import scanpy as sc
import glob
import pandas as pd
import numpy as np
from concurrent.futures import ThreadPoolExecutor, as_completed
from joblib import Parallel, delayed
import argparse
In [11]:
result_dir = "/data/sarkar_lab/Projects/spindle_dev/results/hbrain_10X/"
Path(result_dir).mkdir(parents=True, exist_ok=True)
In [12]:
all_genes = True
resolution=0.5
min_final_size=10
start_time = time.time()
coords = adata.obsm["spatial"]
tiles = preprocessing.build_quadtree_tiles(coords, max_pts=200, min_side=0.0, max_depth=40)
# remove tiles with less than 5 spots
tiles = [tile for tile in tiles if len(tile.idx) >= 5]
tiles = preprocessing.reindex_tiles(tiles)
num_genes = adata.n_vars
genes_work, gene_idx = spindle_dev.preprocessing.topvar_genes(adata, G=num_genes)
tile_covs = spindle_dev.preprocessing.build_tile_covs_full_serial(adata, tiles, gene_idx, eps=1e-6)
data = index.ProcessedData(tiles, tile_covs, genes_work, adata.n_obs)
if 'pca' not in data.latent:
data.reduce_dim(num_pca_components=30, n_components=2, do_umap=True)
data.cluster_spds(cluster_distance="tree", cluster_method="leiden", resolution=resolution)
data.assign_label_to_spots()
data.get_corr_mean_by_cluster()
out_dict = data.get_adaptive_runs(find_blocks=True, with_size_guard=True,min_final_size=min_final_size,max_final_size=100)
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
[2026-01-21 04:07:10,827] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-21 04:07:11,016] INFO spindle_dev.index: Building ultrametric features from SPD matrices. /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/scipy/cluster/hierarchy.py:1673: RuntimeWarning: invalid value encountered in scalar divide c = xp.sum(numerator) / xp.sqrt(xp.sum(denomA) * xp.sum(denomB)) [2026-01-21 04:10:32,508] INFO spindle_dev.index: Computing latent features from the tree representations. [2026-01-21 04:10:56,656] INFO spindle_dev.index: Reducing latent features to 30 dimensions using PCA. [2026-01-21 04:11:42,621] INFO spindle_dev.index: Explained variance ratios by PCA components: [0.03411894 0.02578561 0.01285703 0.00938737 0.00588084 0.00551661 0.00453925 0.00377464 0.00356478 0.00307318 0.00273169 0.0026462 0.00263546 0.00248134 0.00241321 0.00210543 0.00208783 0.00205712 0.0020324 0.00196088 0.00191746 0.00189922 0.00186625 0.00184357 0.00182782 0.00181393 0.00176957 0.00176038 0.00173033 0.00172268] [2026-01-21 04:11:42,626] INFO spindle_dev.index: Reducing latent features to 2 dimensions using UMAP. /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/umap/umap_.py:1952: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism. warn( [2026-01-21 04:12:11,404] INFO spindle_dev.index: Clustering SPD-s using 'tree' distance. [2026-01-21 04:12:11,407] INFO spindle_dev.index: Clustering SPD matrices using Leiden clustering with resolution 0.50. [2026-01-21 04:12:18,224] INFO spindle_dev.index: Since clustering method is tree, I am going to find global order per cluster [2026-01-21 04:12:18,226] INFO spindle_dev.index: Finding consensus tree for cluster 0 [2026-01-21 04:12:23,367] INFO spindle_dev.index: Finding consensus tree for cluster 1 [2026-01-21 04:12:26,198] INFO spindle_dev.index: Finding consensus tree for cluster 2 [2026-01-21 04:12:28,478] INFO spindle_dev.index: Finding consensus tree for cluster 3 [2026-01-21 04:12:30,273] INFO spindle_dev.index: Finding consensus tree for cluster 4 [2026-01-21 04:12:31,280] INFO spindle_dev.index: Finding consensus tree for cluster 5 [2026-01-21 04:12:31,650] INFO spindle_dev.index: Finding consensus tree for cluster 6 [2026-01-21 04:12:31,990] INFO spindle_dev.index: Finding consensus tree for cluster 7 [2026-01-21 04:13:00,643] INFO spindle_dev.index: Computing mean correlation matrix for cluster 0 [2026-01-21 04:13:16,509] INFO spindle_dev.index: Computing mean correlation matrix for cluster 1 [2026-01-21 04:13:30,986] INFO spindle_dev.index: Computing mean correlation matrix for cluster 2 [2026-01-21 04:13:39,913] INFO spindle_dev.index: Computing mean correlation matrix for cluster 3 [2026-01-21 04:13:46,711] INFO spindle_dev.index: Computing mean correlation matrix for cluster 4 [2026-01-21 04:13:53,080] INFO spindle_dev.index: Computing mean correlation matrix for cluster 5 [2026-01-21 04:14:01,600] INFO spindle_dev.index: Computing mean correlation matrix for cluster 6 [2026-01-21 04:14:12,773] INFO spindle_dev.index: Computing mean correlation matrix for cluster 7 [2026-01-21 04:14:13,662] INFO spindle_dev.index: Finding adaptive block runs for cluster 0 [2026-01-21 04:14:14,813] INFO spindle_dev.index: Chose t=0.8632246875507528 resulting in 354 blocks instead of 242 blocks would have gotten by default [2026-01-21 04:14:14,973] INFO spindle_dev.index: Final block runs for cluster 0: 36 blocks. [2026-01-21 04:14:14,974] INFO spindle_dev.index: Finding adaptive block runs for cluster 1 [2026-01-21 04:14:16,060] INFO spindle_dev.index: Chose t=0.8421314547071789 resulting in 370 blocks instead of 245 blocks would have gotten by default [2026-01-21 04:14:16,231] INFO spindle_dev.index: Final block runs for cluster 1: 36 blocks. [2026-01-21 04:14:16,231] INFO spindle_dev.index: Finding adaptive block runs for cluster 2 [2026-01-21 04:14:17,335] INFO spindle_dev.index: Chose t=0.8416540402268181 resulting in 331 blocks instead of 226 blocks would have gotten by default [2026-01-21 04:14:17,477] INFO spindle_dev.index: Final block runs for cluster 2: 35 blocks. [2026-01-21 04:14:17,478] INFO spindle_dev.index: Finding adaptive block runs for cluster 3 [2026-01-21 04:14:18,575] INFO spindle_dev.index: Chose t=0.8631705570040082 resulting in 387 blocks instead of 270 blocks would have gotten by default [2026-01-21 04:14:18,760] INFO spindle_dev.index: Final block runs for cluster 3: 38 blocks. [2026-01-21 04:14:18,760] INFO spindle_dev.index: Finding adaptive block runs for cluster 4 [2026-01-21 04:14:19,860] INFO spindle_dev.index: Chose t=0.784655953570506 resulting in 383 blocks instead of 205 blocks would have gotten by default [2026-01-21 04:14:20,041] INFO spindle_dev.index: Final block runs for cluster 4: 39 blocks. [2026-01-21 04:14:20,042] INFO spindle_dev.index: Finding adaptive block runs for cluster 5 [2026-01-21 04:14:21,137] INFO spindle_dev.index: Chose t=0.8552197331616531 resulting in 326 blocks instead of 208 blocks would have gotten by default [2026-01-21 04:14:21,270] INFO spindle_dev.index: Final block runs for cluster 5: 35 blocks. [2026-01-21 04:14:21,271] INFO spindle_dev.index: Finding adaptive block runs for cluster 6 [2026-01-21 04:14:22,378] INFO spindle_dev.index: Chose t=0.8304446409718929 resulting in 368 blocks instead of 241 blocks would have gotten by default [2026-01-21 04:14:22,544] INFO spindle_dev.index: Final block runs for cluster 6: 38 blocks. [2026-01-21 04:14:22,545] INFO spindle_dev.index: Finding adaptive block runs for cluster 7 [2026-01-21 04:14:23,026] INFO spindle_dev.index: Chose t=0.9705885179838508 resulting in 339 blocks instead of 459 blocks would have gotten by default [2026-01-21 04:14:23,169] INFO spindle_dev.index: Final block runs for cluster 7: 19 blocks.
In [13]:
import matplotlib.pylab as plt
from matplotlib.patches import Rectangle
from matplotlib.collections import PatchCollection
In [14]:
import numpy as np
palette = sc.pl.palettes.default_20
point_colors = [palette[lab] for lab in data.spot_label.values()]
labels = data.labels
indices = np.array([int(i) for i in data.spot_label.keys()])
coords = adata.obsm['spatial']
plt.scatter(coords[indices,0], coords[indices,1], s=1, c=point_colors, lw=0, alpha=0.8)
patches = []
for t in tiles:
bbox = t.bbox if hasattr(t, "bbox") else t["bbox"] # (xmin,ymin,xmax,ymax)
x0, y0, x1, y1 = bbox
patches.append(Rectangle((x0, y0), x1 - x0, y1 - y0, fill=False))
pc = PatchCollection(patches, match_original=True, linewidths=0.2, edgecolors='b', alpha=0.3)
# add collection
ax = plt.gca()
ax.add_collection(pc)
ax.invert_yaxis()
ax.axis('off')
#ax.set_aspect('equal')
plt.savefig(f"{result_dir}/spatial_plot_with_cluster.png", dpi=300)
plt.show()
In [ ]:
In [15]:
# Create colors for clusters
import numpy as np
import matplotlib.pyplot as plt
n_clusters = len(set(data.labels))
# Use Scanpy's default_20 palette for consistency
palette = sc.pl.palettes.default_20
cluster_colors = [palette[lab] for lab in data.labels]
plt.scatter(data.latent['umap'][:,0], data.latent['umap'][:,1], s=6, lw=0, alpha=0.8, c=cluster_colors)
# Write label on the plot
# with backgrond circle for better visibility
# and bold font
for i in range(n_clusters):
cluster_points = data.latent['umap'][data.labels == i]
if len(cluster_points) == 0:
continue
x_mean = np.mean(cluster_points[:,0])
y_mean = np.mean(cluster_points[:,1])
plt.text(x_mean, y_mean, str(i), color='black', fontsize=10, ha='center', va='center', fontweight='bold',
bbox=dict(facecolor='white', edgecolor='black', boxstyle='circle,pad=0.5', alpha=0.7))
# take away top and right border
ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
# remove axis ticks
ax.set_xticks([])
ax.set_yticks([])
plt.savefig(f"{result_dir}/umap_plot_with_cluster_labels.png", dpi=300)
plt.show()
In [16]:
epsilon_block_wise_dict = {}
epsilon_dict = {}
for cluster_id in set(data.labels):
eps_per_block, eps_elbow_per_block, eps = index.choose_adaptive_epsilons(data, cluster_id, k_target_per_block=64)
epsilon_block_wise_dict[int(cluster_id)] = eps_elbow_per_block
epsilon_dict[int(cluster_id)] = eps
In [18]:
epsilon_dict
Out[18]:
{0: np.float64(6.102528176453383),
1: np.float64(7.198015254724661),
2: np.float64(7.622794235690398),
3: np.float64(7.687026962565294),
4: np.float64(6.851659581044885),
5: np.float64(6.414602220897768),
6: np.float64(7.348541690464721),
7: np.float64(1.8774843590512265)}
In [19]:
config = typing.IndexConfig()
config.epsilon_dict = epsilon_dict
config.epsilon_block_wise_dict = epsilon_block_wise_dict
config.threshold_type = 'constant'
config.kmean_method = 'epsilon_net'
In [20]:
# Create index
dag_dict, stat, dist_list = index.index_spds(data, config=config)
[2026-01-21 04:26:48,811] INFO spindle_dev.index: Processing cluster 0 [2026-01-21 04:26:48,813] INFO spindle_dev.index: Building SPD index with epsilon=6.102528176453383 [2026-01-21 04:26:48,817] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:26:49,859] INFO spindle_dev.index: Cluster 0: 3053 SPDs, 36 blocks [2026-01-21 04:26:49,860] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:26:50,083] INFO spindle_dev.index: Finished block 0 in 0.22 seconds, found 2 clusters. [2026-01-21 04:26:50,085] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:26:50,445] INFO spindle_dev.index: Finished block 1 in 0.36 seconds, found 6 clusters. [2026-01-21 04:26:50,446] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:26:50,736] INFO spindle_dev.index: Finished block 2 in 0.29 seconds, found 4 clusters. [2026-01-21 04:26:50,738] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:26:51,029] INFO spindle_dev.index: Finished block 3 in 0.29 seconds, found 4 clusters. [2026-01-21 04:26:51,031] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:26:51,574] INFO spindle_dev.index: Finished block 4 in 0.54 seconds, found 10 clusters. [2026-01-21 04:26:51,576] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:26:52,177] INFO spindle_dev.index: Finished block 5 in 0.60 seconds, found 10 clusters. [2026-01-21 04:26:52,178] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:26:52,760] INFO spindle_dev.index: Finished block 6 in 0.58 seconds, found 11 clusters. [2026-01-21 04:26:52,761] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:26:53,344] INFO spindle_dev.index: Finished block 7 in 0.58 seconds, found 11 clusters. [2026-01-21 04:26:53,345] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:26:53,949] INFO spindle_dev.index: Finished block 8 in 0.60 seconds, found 12 clusters. [2026-01-21 04:26:53,951] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:26:54,548] INFO spindle_dev.index: Finished block 9 in 0.60 seconds, found 12 clusters. [2026-01-21 04:26:54,550] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:26:55,232] INFO spindle_dev.index: Finished block 10 in 0.68 seconds, found 15 clusters. [2026-01-21 04:26:55,234] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:26:55,739] INFO spindle_dev.index: Finished block 11 in 0.51 seconds, found 10 clusters. [2026-01-21 04:26:55,740] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:26:56,406] INFO spindle_dev.index: Finished block 12 in 0.67 seconds, found 15 clusters. [2026-01-21 04:26:56,407] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:26:57,156] INFO spindle_dev.index: Finished block 13 in 0.75 seconds, found 17 clusters. [2026-01-21 04:26:57,157] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:26:57,821] INFO spindle_dev.index: Finished block 14 in 0.66 seconds, found 15 clusters. [2026-01-21 04:26:57,823] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:26:58,543] INFO spindle_dev.index: Finished block 15 in 0.72 seconds, found 17 clusters. [2026-01-21 04:26:58,544] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:26:59,478] INFO spindle_dev.index: Finished block 16 in 0.93 seconds, found 22 clusters. [2026-01-21 04:26:59,479] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:27:00,294] INFO spindle_dev.index: Finished block 17 in 0.81 seconds, found 19 clusters. [2026-01-21 04:27:00,295] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:27:01,208] INFO spindle_dev.index: Finished block 18 in 0.91 seconds, found 20 clusters. [2026-01-21 04:27:01,210] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:27:02,146] INFO spindle_dev.index: Finished block 19 in 0.94 seconds, found 21 clusters. [2026-01-21 04:27:02,147] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:27:03,346] INFO spindle_dev.index: Finished block 20 in 1.20 seconds, found 26 clusters. [2026-01-21 04:27:03,348] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:27:03,887] INFO spindle_dev.index: Finished block 21 in 0.54 seconds, found 10 clusters. [2026-01-21 04:27:03,888] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:27:04,463] INFO spindle_dev.index: Finished block 22 in 0.57 seconds, found 11 clusters. [2026-01-21 04:27:04,464] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:27:05,835] INFO spindle_dev.index: Finished block 23 in 1.37 seconds, found 15 clusters. [2026-01-21 04:27:05,836] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:27:06,351] INFO spindle_dev.index: Finished block 24 in 0.52 seconds, found 6 clusters. [2026-01-21 04:27:06,352] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:27:06,980] INFO spindle_dev.index: Finished block 25 in 0.63 seconds, found 2 clusters. [2026-01-21 04:27:06,981] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:27:07,459] INFO spindle_dev.index: Finished block 26 in 0.48 seconds, found 8 clusters. [2026-01-21 04:27:07,460] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:27:07,708] INFO spindle_dev.index: Finished block 27 in 0.25 seconds, found 2 clusters. [2026-01-21 04:27:07,709] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:27:07,916] INFO spindle_dev.index: Finished block 28 in 0.21 seconds, found 1 clusters. [2026-01-21 04:27:07,917] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:27:08,111] INFO spindle_dev.index: Finished block 29 in 0.19 seconds, found 1 clusters. [2026-01-21 04:27:08,112] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:27:08,318] INFO spindle_dev.index: Finished block 30 in 0.21 seconds, found 1 clusters. [2026-01-21 04:27:08,319] INFO spindle_dev.index: Using epsilon-net clustering for block 31 [2026-01-21 04:27:08,524] INFO spindle_dev.index: Finished block 31 in 0.20 seconds, found 1 clusters. [2026-01-21 04:27:08,524] INFO spindle_dev.index: Using epsilon-net clustering for block 32 [2026-01-21 04:27:08,731] INFO spindle_dev.index: Finished block 32 in 0.21 seconds, found 1 clusters. [2026-01-21 04:27:08,732] INFO spindle_dev.index: Using epsilon-net clustering for block 33 [2026-01-21 04:27:08,941] INFO spindle_dev.index: Finished block 33 in 0.21 seconds, found 1 clusters. [2026-01-21 04:27:08,942] INFO spindle_dev.index: Using epsilon-net clustering for block 34 [2026-01-21 04:27:09,174] INFO spindle_dev.index: Finished block 34 in 0.23 seconds, found 1 clusters. [2026-01-21 04:27:09,175] INFO spindle_dev.index: Using epsilon-net clustering for block 35 [2026-01-21 04:27:10,225] INFO spindle_dev.index: Finished block 35 in 1.05 seconds, found 1 clusters. [2026-01-21 04:27:10,226] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:27:10,227] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:27:10,227] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:27:10,228] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:27:10,229] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:27:10,311] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:27:10,312] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:27:10,313] INFO spindle_dev.index: Processing cluster 1 [2026-01-21 04:27:10,313] INFO spindle_dev.index: Building SPD index with epsilon=7.198015254724661 [2026-01-21 04:27:10,314] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:27:11,326] INFO spindle_dev.index: Cluster 1: 2643 SPDs, 36 blocks [2026-01-21 04:27:11,329] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:27:11,493] INFO spindle_dev.index: Finished block 0 in 0.16 seconds, found 1 clusters. [2026-01-21 04:27:11,494] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:27:11,636] INFO spindle_dev.index: Finished block 1 in 0.14 seconds, found 1 clusters. [2026-01-21 04:27:11,637] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:27:11,805] INFO spindle_dev.index: Finished block 2 in 0.17 seconds, found 2 clusters. [2026-01-21 04:27:11,806] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:27:11,981] INFO spindle_dev.index: Finished block 3 in 0.17 seconds, found 2 clusters. [2026-01-21 04:27:11,983] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:27:12,199] INFO spindle_dev.index: Finished block 4 in 0.22 seconds, found 3 clusters. [2026-01-21 04:27:12,200] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:27:12,457] INFO spindle_dev.index: Finished block 5 in 0.26 seconds, found 4 clusters. [2026-01-21 04:27:12,459] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:27:12,704] INFO spindle_dev.index: Finished block 6 in 0.25 seconds, found 3 clusters. [2026-01-21 04:27:12,705] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:27:12,928] INFO spindle_dev.index: Finished block 7 in 0.22 seconds, found 2 clusters. [2026-01-21 04:27:12,930] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:27:13,231] INFO spindle_dev.index: Finished block 8 in 0.30 seconds, found 4 clusters. [2026-01-21 04:27:13,233] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:27:13,473] INFO spindle_dev.index: Finished block 9 in 0.24 seconds, found 2 clusters. [2026-01-21 04:27:13,475] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:27:13,831] INFO spindle_dev.index: Finished block 10 in 0.36 seconds, found 3 clusters. [2026-01-21 04:27:13,832] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:27:14,128] INFO spindle_dev.index: Finished block 11 in 0.30 seconds, found 3 clusters. [2026-01-21 04:27:14,131] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:27:14,458] INFO spindle_dev.index: Finished block 12 in 0.33 seconds, found 3 clusters. [2026-01-21 04:27:14,459] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:27:14,838] INFO spindle_dev.index: Finished block 13 in 0.38 seconds, found 3 clusters. [2026-01-21 04:27:14,840] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:27:15,217] INFO spindle_dev.index: Finished block 14 in 0.38 seconds, found 3 clusters. [2026-01-21 04:27:15,219] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:27:15,506] INFO spindle_dev.index: Finished block 15 in 0.29 seconds, found 2 clusters. [2026-01-21 04:27:15,508] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:27:15,895] INFO spindle_dev.index: Finished block 16 in 0.39 seconds, found 4 clusters. [2026-01-21 04:27:15,897] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:27:16,354] INFO spindle_dev.index: Finished block 17 in 0.46 seconds, found 6 clusters. [2026-01-21 04:27:16,355] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:27:16,763] INFO spindle_dev.index: Finished block 18 in 0.41 seconds, found 6 clusters. [2026-01-21 04:27:16,765] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:27:17,186] INFO spindle_dev.index: Finished block 19 in 0.42 seconds, found 7 clusters. [2026-01-21 04:27:17,187] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:27:17,428] INFO spindle_dev.index: Finished block 20 in 0.24 seconds, found 2 clusters. [2026-01-21 04:27:17,429] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:27:17,820] INFO spindle_dev.index: Finished block 21 in 0.39 seconds, found 6 clusters. [2026-01-21 04:27:17,822] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:27:18,155] INFO spindle_dev.index: Finished block 22 in 0.33 seconds, found 5 clusters. [2026-01-21 04:27:18,156] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:27:18,445] INFO spindle_dev.index: Finished block 23 in 0.29 seconds, found 3 clusters. [2026-01-21 04:27:18,447] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:27:18,710] INFO spindle_dev.index: Finished block 24 in 0.26 seconds, found 2 clusters. [2026-01-21 04:27:18,711] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:27:19,939] INFO spindle_dev.index: Finished block 25 in 1.23 seconds, found 5 clusters. [2026-01-21 04:27:19,940] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:27:20,255] INFO spindle_dev.index: Finished block 26 in 0.32 seconds, found 5 clusters. [2026-01-21 04:27:20,257] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:27:20,645] INFO spindle_dev.index: Finished block 27 in 0.39 seconds, found 4 clusters. [2026-01-21 04:27:20,646] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:27:20,879] INFO spindle_dev.index: Finished block 28 in 0.23 seconds, found 2 clusters. [2026-01-21 04:27:20,880] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:27:21,088] INFO spindle_dev.index: Finished block 29 in 0.21 seconds, found 1 clusters. [2026-01-21 04:27:21,090] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:27:21,300] INFO spindle_dev.index: Finished block 30 in 0.21 seconds, found 1 clusters. [2026-01-21 04:27:21,301] INFO spindle_dev.index: Using epsilon-net clustering for block 31 [2026-01-21 04:27:21,549] INFO spindle_dev.index: Finished block 31 in 0.25 seconds, found 1 clusters. [2026-01-21 04:27:21,550] INFO spindle_dev.index: Using epsilon-net clustering for block 32 [2026-01-21 04:27:21,776] INFO spindle_dev.index: Finished block 32 in 0.23 seconds, found 2 clusters. [2026-01-21 04:27:21,778] INFO spindle_dev.index: Using epsilon-net clustering for block 33 [2026-01-21 04:27:21,964] INFO spindle_dev.index: Finished block 33 in 0.19 seconds, found 1 clusters. [2026-01-21 04:27:21,965] INFO spindle_dev.index: Using epsilon-net clustering for block 34 [2026-01-21 04:27:22,175] INFO spindle_dev.index: Finished block 34 in 0.21 seconds, found 1 clusters. [2026-01-21 04:27:22,176] INFO spindle_dev.index: Using epsilon-net clustering for block 35 [2026-01-21 04:27:24,229] INFO spindle_dev.index: Finished block 35 in 2.05 seconds, found 26 clusters. [2026-01-21 04:27:24,231] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:27:24,232] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:27:24,232] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:27:24,233] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:27:24,234] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:27:24,948] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:27:24,949] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:27:24,950] INFO spindle_dev.index: Processing cluster 2 [2026-01-21 04:27:24,950] INFO spindle_dev.index: Building SPD index with epsilon=7.622794235690398 [2026-01-21 04:27:24,951] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:27:25,540] INFO spindle_dev.index: Cluster 2: 1523 SPDs, 35 blocks [2026-01-21 04:27:25,542] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:27:25,639] INFO spindle_dev.index: Finished block 0 in 0.10 seconds, found 1 clusters. [2026-01-21 04:27:25,639] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:27:25,726] INFO spindle_dev.index: Finished block 1 in 0.09 seconds, found 1 clusters. [2026-01-21 04:27:25,727] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:27:25,813] INFO spindle_dev.index: Finished block 2 in 0.09 seconds, found 1 clusters. [2026-01-21 04:27:25,814] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:27:25,918] INFO spindle_dev.index: Finished block 3 in 0.10 seconds, found 2 clusters. [2026-01-21 04:27:25,919] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:27:26,007] INFO spindle_dev.index: Finished block 4 in 0.09 seconds, found 1 clusters. [2026-01-21 04:27:26,008] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:27:26,110] INFO spindle_dev.index: Finished block 5 in 0.10 seconds, found 2 clusters. [2026-01-21 04:27:26,111] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:27:26,220] INFO spindle_dev.index: Finished block 6 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:26,221] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:27:26,336] INFO spindle_dev.index: Finished block 7 in 0.12 seconds, found 2 clusters. [2026-01-21 04:27:26,337] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:27:26,463] INFO spindle_dev.index: Finished block 8 in 0.13 seconds, found 3 clusters. [2026-01-21 04:27:26,464] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:27:26,591] INFO spindle_dev.index: Finished block 9 in 0.13 seconds, found 3 clusters. [2026-01-21 04:27:26,592] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:27:26,720] INFO spindle_dev.index: Finished block 10 in 0.13 seconds, found 3 clusters. [2026-01-21 04:27:26,721] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:27:26,846] INFO spindle_dev.index: Finished block 11 in 0.13 seconds, found 3 clusters. [2026-01-21 04:27:26,847] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:27:26,949] INFO spindle_dev.index: Finished block 12 in 0.10 seconds, found 2 clusters. [2026-01-21 04:27:26,949] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:27:27,064] INFO spindle_dev.index: Finished block 13 in 0.11 seconds, found 3 clusters. [2026-01-21 04:27:27,065] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:27:27,188] INFO spindle_dev.index: Finished block 14 in 0.12 seconds, found 3 clusters. [2026-01-21 04:27:27,189] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:27:27,322] INFO spindle_dev.index: Finished block 15 in 0.13 seconds, found 4 clusters. [2026-01-21 04:27:27,323] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:27:27,439] INFO spindle_dev.index: Finished block 16 in 0.12 seconds, found 3 clusters. [2026-01-21 04:27:27,439] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:27:27,600] INFO spindle_dev.index: Finished block 17 in 0.16 seconds, found 5 clusters. [2026-01-21 04:27:27,601] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:27:27,754] INFO spindle_dev.index: Finished block 18 in 0.15 seconds, found 4 clusters. [2026-01-21 04:27:27,755] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:27:27,944] INFO spindle_dev.index: Finished block 19 in 0.19 seconds, found 6 clusters. [2026-01-21 04:27:27,944] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:27:28,099] INFO spindle_dev.index: Finished block 20 in 0.15 seconds, found 4 clusters. [2026-01-21 04:27:28,100] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:27:28,278] INFO spindle_dev.index: Finished block 21 in 0.18 seconds, found 5 clusters. [2026-01-21 04:27:28,279] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:27:28,455] INFO spindle_dev.index: Finished block 22 in 0.18 seconds, found 5 clusters. [2026-01-21 04:27:28,456] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:27:28,594] INFO spindle_dev.index: Finished block 23 in 0.14 seconds, found 3 clusters. [2026-01-21 04:27:28,595] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:27:29,170] INFO spindle_dev.index: Finished block 24 in 0.57 seconds, found 6 clusters. [2026-01-21 04:27:29,171] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:27:29,341] INFO spindle_dev.index: Finished block 25 in 0.17 seconds, found 3 clusters. [2026-01-21 04:27:29,342] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:27:29,465] INFO spindle_dev.index: Finished block 26 in 0.12 seconds, found 2 clusters. [2026-01-21 04:27:29,466] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:27:29,608] INFO spindle_dev.index: Finished block 27 in 0.14 seconds, found 3 clusters. [2026-01-21 04:27:29,609] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:27:29,733] INFO spindle_dev.index: Finished block 28 in 0.12 seconds, found 2 clusters. [2026-01-21 04:27:29,734] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:27:29,859] INFO spindle_dev.index: Finished block 29 in 0.12 seconds, found 2 clusters. [2026-01-21 04:27:29,860] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:27:29,983] INFO spindle_dev.index: Finished block 30 in 0.12 seconds, found 2 clusters. [2026-01-21 04:27:29,984] INFO spindle_dev.index: Using epsilon-net clustering for block 31 [2026-01-21 04:27:30,106] INFO spindle_dev.index: Finished block 31 in 0.12 seconds, found 2 clusters. [2026-01-21 04:27:30,107] INFO spindle_dev.index: Using epsilon-net clustering for block 32 [2026-01-21 04:27:30,901] INFO spindle_dev.index: Finished block 32 in 0.79 seconds, found 14 clusters. [2026-01-21 04:27:30,902] INFO spindle_dev.index: Using epsilon-net clustering for block 33 [2026-01-21 04:27:31,090] INFO spindle_dev.index: Finished block 33 in 0.19 seconds, found 4 clusters. [2026-01-21 04:27:31,091] INFO spindle_dev.index: Using epsilon-net clustering for block 34 [2026-01-21 04:27:31,615] INFO spindle_dev.index: Finished block 34 in 0.52 seconds, found 2 clusters. [2026-01-21 04:27:31,616] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:27:31,617] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:27:31,617] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:27:31,618] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:27:31,618] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:27:31,658] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:27:31,659] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:27:31,660] INFO spindle_dev.index: Processing cluster 3 [2026-01-21 04:27:31,660] INFO spindle_dev.index: Building SPD index with epsilon=7.687026962565294 [2026-01-21 04:27:31,661] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:27:32,233] INFO spindle_dev.index: Cluster 3: 1479 SPDs, 38 blocks [2026-01-21 04:27:32,234] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:27:32,326] INFO spindle_dev.index: Finished block 0 in 0.09 seconds, found 1 clusters. [2026-01-21 04:27:32,327] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:27:32,413] INFO spindle_dev.index: Finished block 1 in 0.09 seconds, found 1 clusters. [2026-01-21 04:27:32,414] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:27:32,500] INFO spindle_dev.index: Finished block 2 in 0.09 seconds, found 1 clusters. [2026-01-21 04:27:32,501] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:27:32,595] INFO spindle_dev.index: Finished block 3 in 0.09 seconds, found 2 clusters. [2026-01-21 04:27:32,596] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:27:32,672] INFO spindle_dev.index: Finished block 4 in 0.08 seconds, found 1 clusters. [2026-01-21 04:27:32,673] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:27:32,764] INFO spindle_dev.index: Finished block 5 in 0.09 seconds, found 2 clusters. [2026-01-21 04:27:32,765] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:27:32,844] INFO spindle_dev.index: Finished block 6 in 0.08 seconds, found 1 clusters. [2026-01-21 04:27:32,845] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:27:32,937] INFO spindle_dev.index: Finished block 7 in 0.09 seconds, found 2 clusters. [2026-01-21 04:27:32,938] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:27:33,031] INFO spindle_dev.index: Finished block 8 in 0.09 seconds, found 2 clusters. [2026-01-21 04:27:33,032] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:27:33,143] INFO spindle_dev.index: Finished block 9 in 0.11 seconds, found 1 clusters. [2026-01-21 04:27:33,144] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:27:33,243] INFO spindle_dev.index: Finished block 10 in 0.10 seconds, found 2 clusters. [2026-01-21 04:27:33,244] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:27:33,350] INFO spindle_dev.index: Finished block 11 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:33,351] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:27:33,458] INFO spindle_dev.index: Finished block 12 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:33,459] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:27:33,570] INFO spindle_dev.index: Finished block 13 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:33,571] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:27:33,678] INFO spindle_dev.index: Finished block 14 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:33,678] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:27:33,784] INFO spindle_dev.index: Finished block 15 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:33,785] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:27:33,892] INFO spindle_dev.index: Finished block 16 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:33,893] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:27:34,004] INFO spindle_dev.index: Finished block 17 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:34,005] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:27:34,115] INFO spindle_dev.index: Finished block 18 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:34,116] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:27:34,226] INFO spindle_dev.index: Finished block 19 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:34,227] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:27:34,392] INFO spindle_dev.index: Finished block 20 in 0.17 seconds, found 5 clusters. [2026-01-21 04:27:34,393] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:27:34,540] INFO spindle_dev.index: Finished block 21 in 0.15 seconds, found 3 clusters. [2026-01-21 04:27:34,541] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:27:34,707] INFO spindle_dev.index: Finished block 22 in 0.17 seconds, found 3 clusters. [2026-01-21 04:27:34,708] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:27:34,880] INFO spindle_dev.index: Finished block 23 in 0.17 seconds, found 5 clusters. [2026-01-21 04:27:34,881] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:27:35,027] INFO spindle_dev.index: Finished block 24 in 0.15 seconds, found 4 clusters. [2026-01-21 04:27:35,028] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:27:35,200] INFO spindle_dev.index: Finished block 25 in 0.17 seconds, found 5 clusters. [2026-01-21 04:27:35,201] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:27:35,391] INFO spindle_dev.index: Finished block 26 in 0.19 seconds, found 6 clusters. [2026-01-21 04:27:35,392] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:27:35,564] INFO spindle_dev.index: Finished block 27 in 0.17 seconds, found 5 clusters. [2026-01-21 04:27:35,565] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:27:35,737] INFO spindle_dev.index: Finished block 28 in 0.17 seconds, found 5 clusters. [2026-01-21 04:27:35,737] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:27:35,912] INFO spindle_dev.index: Finished block 29 in 0.17 seconds, found 5 clusters. [2026-01-21 04:27:35,913] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:27:36,073] INFO spindle_dev.index: Finished block 30 in 0.16 seconds, found 4 clusters. [2026-01-21 04:27:36,074] INFO spindle_dev.index: Using epsilon-net clustering for block 31 [2026-01-21 04:27:36,267] INFO spindle_dev.index: Finished block 31 in 0.19 seconds, found 6 clusters. [2026-01-21 04:27:36,268] INFO spindle_dev.index: Using epsilon-net clustering for block 32 [2026-01-21 04:27:36,445] INFO spindle_dev.index: Finished block 32 in 0.18 seconds, found 5 clusters. [2026-01-21 04:27:36,446] INFO spindle_dev.index: Using epsilon-net clustering for block 33 [2026-01-21 04:27:36,749] INFO spindle_dev.index: Finished block 33 in 0.30 seconds, found 9 clusters. [2026-01-21 04:27:36,750] INFO spindle_dev.index: Using epsilon-net clustering for block 34 [2026-01-21 04:27:37,785] INFO spindle_dev.index: Finished block 34 in 1.03 seconds, found 22 clusters. [2026-01-21 04:27:37,786] INFO spindle_dev.index: Using epsilon-net clustering for block 35 [2026-01-21 04:27:37,923] INFO spindle_dev.index: Finished block 35 in 0.14 seconds, found 3 clusters. [2026-01-21 04:27:37,924] INFO spindle_dev.index: Using epsilon-net clustering for block 36 [2026-01-21 04:27:38,078] INFO spindle_dev.index: Finished block 36 in 0.15 seconds, found 4 clusters. [2026-01-21 04:27:38,079] INFO spindle_dev.index: Using epsilon-net clustering for block 37 [2026-01-21 04:27:38,449] INFO spindle_dev.index: Finished block 37 in 0.37 seconds, found 3 clusters. [2026-01-21 04:27:38,450] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:27:38,451] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:27:38,451] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:27:38,451] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:27:38,452] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:27:38,493] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:27:38,494] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:27:38,495] INFO spindle_dev.index: Processing cluster 4 [2026-01-21 04:27:38,495] INFO spindle_dev.index: Building SPD index with epsilon=6.851659581044885 [2026-01-21 04:27:38,496] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:27:38,849] INFO spindle_dev.index: Cluster 4: 1196 SPDs, 39 blocks [2026-01-21 04:27:38,851] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:27:38,921] INFO spindle_dev.index: Finished block 0 in 0.07 seconds, found 1 clusters. [2026-01-21 04:27:38,922] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:27:39,022] INFO spindle_dev.index: Finished block 1 in 0.10 seconds, found 4 clusters. [2026-01-21 04:27:39,023] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:27:39,124] INFO spindle_dev.index: Finished block 2 in 0.10 seconds, found 4 clusters. [2026-01-21 04:27:39,125] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:27:39,252] INFO spindle_dev.index: Finished block 3 in 0.13 seconds, found 5 clusters. [2026-01-21 04:27:39,253] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:27:39,365] INFO spindle_dev.index: Finished block 4 in 0.11 seconds, found 4 clusters. [2026-01-21 04:27:39,366] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:27:39,515] INFO spindle_dev.index: Finished block 5 in 0.15 seconds, found 6 clusters. [2026-01-21 04:27:39,517] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:27:39,673] INFO spindle_dev.index: Finished block 6 in 0.16 seconds, found 7 clusters. [2026-01-21 04:27:39,674] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:27:39,823] INFO spindle_dev.index: Finished block 7 in 0.15 seconds, found 6 clusters. [2026-01-21 04:27:39,824] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:27:39,972] INFO spindle_dev.index: Finished block 8 in 0.15 seconds, found 6 clusters. [2026-01-21 04:27:39,973] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:27:40,120] INFO spindle_dev.index: Finished block 9 in 0.15 seconds, found 6 clusters. [2026-01-21 04:27:40,121] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:27:40,347] INFO spindle_dev.index: Finished block 10 in 0.23 seconds, found 11 clusters. [2026-01-21 04:27:40,349] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:27:40,525] INFO spindle_dev.index: Finished block 11 in 0.18 seconds, found 8 clusters. [2026-01-21 04:27:40,526] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:27:40,704] INFO spindle_dev.index: Finished block 12 in 0.18 seconds, found 8 clusters. [2026-01-21 04:27:40,705] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:27:40,896] INFO spindle_dev.index: Finished block 13 in 0.19 seconds, found 9 clusters. [2026-01-21 04:27:40,896] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:27:41,085] INFO spindle_dev.index: Finished block 14 in 0.19 seconds, found 9 clusters. [2026-01-21 04:27:41,086] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:27:41,309] INFO spindle_dev.index: Finished block 15 in 0.22 seconds, found 11 clusters. [2026-01-21 04:27:41,310] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:27:41,535] INFO spindle_dev.index: Finished block 16 in 0.23 seconds, found 11 clusters. [2026-01-21 04:27:41,536] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:27:41,807] INFO spindle_dev.index: Finished block 17 in 0.27 seconds, found 14 clusters. [2026-01-21 04:27:41,807] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:27:42,106] INFO spindle_dev.index: Finished block 18 in 0.30 seconds, found 13 clusters. [2026-01-21 04:27:42,107] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:27:42,365] INFO spindle_dev.index: Finished block 19 in 0.26 seconds, found 13 clusters. [2026-01-21 04:27:42,365] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:27:42,580] INFO spindle_dev.index: Finished block 20 in 0.21 seconds, found 10 clusters. [2026-01-21 04:27:42,581] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:27:42,756] INFO spindle_dev.index: Finished block 21 in 0.18 seconds, found 7 clusters. [2026-01-21 04:27:42,757] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:27:42,912] INFO spindle_dev.index: Finished block 22 in 0.15 seconds, found 6 clusters. [2026-01-21 04:27:42,912] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:27:43,097] INFO spindle_dev.index: Finished block 23 in 0.18 seconds, found 8 clusters. [2026-01-21 04:27:43,098] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:27:43,224] INFO spindle_dev.index: Finished block 24 in 0.13 seconds, found 4 clusters. [2026-01-21 04:27:43,225] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:27:43,362] INFO spindle_dev.index: Finished block 25 in 0.14 seconds, found 2 clusters. [2026-01-21 04:27:43,362] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:27:43,461] INFO spindle_dev.index: Finished block 26 in 0.10 seconds, found 2 clusters. [2026-01-21 04:27:43,462] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:27:43,559] INFO spindle_dev.index: Finished block 27 in 0.10 seconds, found 2 clusters. [2026-01-21 04:27:43,560] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:27:43,657] INFO spindle_dev.index: Finished block 28 in 0.10 seconds, found 2 clusters. [2026-01-21 04:27:43,658] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:27:43,756] INFO spindle_dev.index: Finished block 29 in 0.10 seconds, found 2 clusters. [2026-01-21 04:27:43,757] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:27:43,869] INFO spindle_dev.index: Finished block 30 in 0.11 seconds, found 2 clusters. [2026-01-21 04:27:43,870] INFO spindle_dev.index: Using epsilon-net clustering for block 31 [2026-01-21 04:27:44,001] INFO spindle_dev.index: Finished block 31 in 0.13 seconds, found 2 clusters. [2026-01-21 04:27:44,002] INFO spindle_dev.index: Using epsilon-net clustering for block 32 [2026-01-21 04:27:44,124] INFO spindle_dev.index: Finished block 32 in 0.12 seconds, found 2 clusters. [2026-01-21 04:27:44,125] INFO spindle_dev.index: Using epsilon-net clustering for block 33 [2026-01-21 04:27:44,221] INFO spindle_dev.index: Finished block 33 in 0.10 seconds, found 2 clusters. [2026-01-21 04:27:44,221] INFO spindle_dev.index: Using epsilon-net clustering for block 34 [2026-01-21 04:27:44,348] INFO spindle_dev.index: Finished block 34 in 0.13 seconds, found 5 clusters. [2026-01-21 04:27:44,348] INFO spindle_dev.index: Using epsilon-net clustering for block 35 [2026-01-21 04:27:44,503] INFO spindle_dev.index: Finished block 35 in 0.15 seconds, found 7 clusters. [2026-01-21 04:27:44,504] INFO spindle_dev.index: Using epsilon-net clustering for block 36 [2026-01-21 04:27:44,577] INFO spindle_dev.index: Finished block 36 in 0.07 seconds, found 1 clusters. [2026-01-21 04:27:44,578] INFO spindle_dev.index: Using epsilon-net clustering for block 37 [2026-01-21 04:27:44,695] INFO spindle_dev.index: Finished block 37 in 0.12 seconds, found 4 clusters. [2026-01-21 04:27:44,696] INFO spindle_dev.index: Using epsilon-net clustering for block 38 [2026-01-21 04:27:45,279] INFO spindle_dev.index: Finished block 38 in 0.58 seconds, found 8 clusters. [2026-01-21 04:27:45,280] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:27:45,280] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:27:45,281] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:27:45,281] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:27:45,282] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:27:45,318] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:27:45,319] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:27:45,319] INFO spindle_dev.index: Processing cluster 5 [2026-01-21 04:27:45,320] INFO spindle_dev.index: Building SPD index with epsilon=6.414602220897768 [2026-01-21 04:27:45,320] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:27:45,406] INFO spindle_dev.index: Cluster 5: 460 SPDs, 35 blocks [2026-01-21 04:27:45,408] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:27:45,436] INFO spindle_dev.index: Finished block 0 in 0.03 seconds, found 1 clusters. [2026-01-21 04:27:45,437] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:27:45,476] INFO spindle_dev.index: Finished block 1 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:45,477] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:27:45,506] INFO spindle_dev.index: Finished block 2 in 0.03 seconds, found 1 clusters. [2026-01-21 04:27:45,507] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:27:45,554] INFO spindle_dev.index: Finished block 3 in 0.05 seconds, found 4 clusters. [2026-01-21 04:27:45,555] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:27:45,607] INFO spindle_dev.index: Finished block 4 in 0.05 seconds, found 5 clusters. [2026-01-21 04:27:45,608] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:27:45,674] INFO spindle_dev.index: Finished block 5 in 0.07 seconds, found 7 clusters. [2026-01-21 04:27:45,675] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:27:45,727] INFO spindle_dev.index: Finished block 6 in 0.05 seconds, found 5 clusters. [2026-01-21 04:27:45,728] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:27:45,794] INFO spindle_dev.index: Finished block 7 in 0.07 seconds, found 7 clusters. [2026-01-21 04:27:45,795] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:27:45,861] INFO spindle_dev.index: Finished block 8 in 0.07 seconds, found 7 clusters. [2026-01-21 04:27:45,862] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:27:45,929] INFO spindle_dev.index: Finished block 9 in 0.07 seconds, found 7 clusters. [2026-01-21 04:27:45,930] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:27:46,016] INFO spindle_dev.index: Finished block 10 in 0.09 seconds, found 10 clusters. [2026-01-21 04:27:46,017] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:27:46,097] INFO spindle_dev.index: Finished block 11 in 0.08 seconds, found 9 clusters. [2026-01-21 04:27:46,098] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:27:46,182] INFO spindle_dev.index: Finished block 12 in 0.08 seconds, found 10 clusters. [2026-01-21 04:27:46,183] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:27:46,270] INFO spindle_dev.index: Finished block 13 in 0.09 seconds, found 11 clusters. [2026-01-21 04:27:46,271] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:27:46,347] INFO spindle_dev.index: Finished block 14 in 0.08 seconds, found 9 clusters. [2026-01-21 04:27:46,347] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:27:46,467] INFO spindle_dev.index: Finished block 15 in 0.12 seconds, found 16 clusters. [2026-01-21 04:27:46,468] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:27:46,571] INFO spindle_dev.index: Finished block 16 in 0.10 seconds, found 14 clusters. [2026-01-21 04:27:46,572] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:27:46,671] INFO spindle_dev.index: Finished block 17 in 0.10 seconds, found 13 clusters. [2026-01-21 04:27:46,671] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:27:46,755] INFO spindle_dev.index: Finished block 18 in 0.08 seconds, found 10 clusters. [2026-01-21 04:27:46,756] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:27:46,812] INFO spindle_dev.index: Finished block 19 in 0.06 seconds, found 5 clusters. [2026-01-21 04:27:46,812] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:27:46,852] INFO spindle_dev.index: Finished block 20 in 0.04 seconds, found 2 clusters. [2026-01-21 04:27:46,852] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:27:47,017] INFO spindle_dev.index: Finished block 21 in 0.16 seconds, found 6 clusters. [2026-01-21 04:27:47,018] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:27:47,083] INFO spindle_dev.index: Finished block 22 in 0.07 seconds, found 6 clusters. [2026-01-21 04:27:47,084] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:27:47,129] INFO spindle_dev.index: Finished block 23 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:47,130] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:27:47,212] INFO spindle_dev.index: Finished block 24 in 0.08 seconds, found 2 clusters. [2026-01-21 04:27:47,213] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:27:47,266] INFO spindle_dev.index: Finished block 25 in 0.05 seconds, found 2 clusters. [2026-01-21 04:27:47,267] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:27:47,301] INFO spindle_dev.index: Finished block 26 in 0.03 seconds, found 1 clusters. [2026-01-21 04:27:47,302] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:27:47,336] INFO spindle_dev.index: Finished block 27 in 0.03 seconds, found 1 clusters. [2026-01-21 04:27:47,337] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:27:47,373] INFO spindle_dev.index: Finished block 28 in 0.04 seconds, found 1 clusters. [2026-01-21 04:27:47,374] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:27:47,409] INFO spindle_dev.index: Finished block 29 in 0.03 seconds, found 1 clusters. [2026-01-21 04:27:47,410] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:27:47,447] INFO spindle_dev.index: Finished block 30 in 0.04 seconds, found 1 clusters. [2026-01-21 04:27:47,448] INFO spindle_dev.index: Using epsilon-net clustering for block 31 [2026-01-21 04:27:47,652] INFO spindle_dev.index: Finished block 31 in 0.20 seconds, found 6 clusters. [2026-01-21 04:27:47,653] INFO spindle_dev.index: Using epsilon-net clustering for block 32 [2026-01-21 04:27:47,752] INFO spindle_dev.index: Finished block 32 in 0.10 seconds, found 11 clusters. [2026-01-21 04:27:47,753] INFO spindle_dev.index: Using epsilon-net clustering for block 33 [2026-01-21 04:27:47,838] INFO spindle_dev.index: Finished block 33 in 0.08 seconds, found 10 clusters. [2026-01-21 04:27:47,839] INFO spindle_dev.index: Using epsilon-net clustering for block 34 [2026-01-21 04:27:48,049] INFO spindle_dev.index: Finished block 34 in 0.21 seconds, found 25 clusters. [2026-01-21 04:27:48,050] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:27:48,050] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:27:48,051] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:27:48,051] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:27:48,052] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:27:48,066] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:27:48,067] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:27:48,068] INFO spindle_dev.index: Processing cluster 6 [2026-01-21 04:27:48,068] INFO spindle_dev.index: Building SPD index with epsilon=7.348541690464721 [2026-01-21 04:27:48,069] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:27:48,146] INFO spindle_dev.index: Cluster 6: 414 SPDs, 38 blocks [2026-01-21 04:27:48,147] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:27:48,174] INFO spindle_dev.index: Finished block 0 in 0.03 seconds, found 1 clusters. [2026-01-21 04:27:48,175] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:27:48,199] INFO spindle_dev.index: Finished block 1 in 0.02 seconds, found 1 clusters. [2026-01-21 04:27:48,200] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:27:48,229] INFO spindle_dev.index: Finished block 2 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:48,230] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:27:48,259] INFO spindle_dev.index: Finished block 3 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:48,260] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:27:48,290] INFO spindle_dev.index: Finished block 4 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:48,291] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:27:48,321] INFO spindle_dev.index: Finished block 5 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:48,321] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:27:48,357] INFO spindle_dev.index: Finished block 6 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:48,358] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:27:48,393] INFO spindle_dev.index: Finished block 7 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:48,394] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:27:48,430] INFO spindle_dev.index: Finished block 8 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:48,431] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:27:48,461] INFO spindle_dev.index: Finished block 9 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:48,462] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:27:48,498] INFO spindle_dev.index: Finished block 10 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:48,499] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:27:48,532] INFO spindle_dev.index: Finished block 11 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:48,532] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:27:48,574] INFO spindle_dev.index: Finished block 12 in 0.04 seconds, found 4 clusters. [2026-01-21 04:27:48,575] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:27:48,613] INFO spindle_dev.index: Finished block 13 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:48,613] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:27:48,661] INFO spindle_dev.index: Finished block 14 in 0.05 seconds, found 5 clusters. [2026-01-21 04:27:48,661] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:27:48,699] INFO spindle_dev.index: Finished block 15 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:48,700] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:27:48,737] INFO spindle_dev.index: Finished block 16 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:48,738] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:27:48,781] INFO spindle_dev.index: Finished block 17 in 0.04 seconds, found 4 clusters. [2026-01-21 04:27:48,782] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:27:48,827] INFO spindle_dev.index: Finished block 18 in 0.05 seconds, found 4 clusters. [2026-01-21 04:27:48,828] INFO spindle_dev.index: Using epsilon-net clustering for block 19 [2026-01-21 04:27:48,888] INFO spindle_dev.index: Finished block 19 in 0.06 seconds, found 7 clusters. [2026-01-21 04:27:48,889] INFO spindle_dev.index: Using epsilon-net clustering for block 20 [2026-01-21 04:27:48,927] INFO spindle_dev.index: Finished block 20 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:48,928] INFO spindle_dev.index: Using epsilon-net clustering for block 21 [2026-01-21 04:27:48,977] INFO spindle_dev.index: Finished block 21 in 0.05 seconds, found 5 clusters. [2026-01-21 04:27:48,977] INFO spindle_dev.index: Using epsilon-net clustering for block 22 [2026-01-21 04:27:49,033] INFO spindle_dev.index: Finished block 22 in 0.06 seconds, found 4 clusters. [2026-01-21 04:27:49,033] INFO spindle_dev.index: Using epsilon-net clustering for block 23 [2026-01-21 04:27:49,072] INFO spindle_dev.index: Finished block 23 in 0.04 seconds, found 4 clusters. [2026-01-21 04:27:49,073] INFO spindle_dev.index: Using epsilon-net clustering for block 24 [2026-01-21 04:27:49,103] INFO spindle_dev.index: Finished block 24 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:49,103] INFO spindle_dev.index: Using epsilon-net clustering for block 25 [2026-01-21 04:27:49,142] INFO spindle_dev.index: Finished block 25 in 0.04 seconds, found 4 clusters. [2026-01-21 04:27:49,143] INFO spindle_dev.index: Using epsilon-net clustering for block 26 [2026-01-21 04:27:49,279] INFO spindle_dev.index: Finished block 26 in 0.14 seconds, found 4 clusters. [2026-01-21 04:27:49,280] INFO spindle_dev.index: Using epsilon-net clustering for block 27 [2026-01-21 04:27:49,321] INFO spindle_dev.index: Finished block 27 in 0.04 seconds, found 4 clusters. [2026-01-21 04:27:49,322] INFO spindle_dev.index: Using epsilon-net clustering for block 28 [2026-01-21 04:27:49,357] INFO spindle_dev.index: Finished block 28 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:49,358] INFO spindle_dev.index: Using epsilon-net clustering for block 29 [2026-01-21 04:27:49,392] INFO spindle_dev.index: Finished block 29 in 0.03 seconds, found 3 clusters. [2026-01-21 04:27:49,392] INFO spindle_dev.index: Using epsilon-net clustering for block 30 [2026-01-21 04:27:49,429] INFO spindle_dev.index: Finished block 30 in 0.04 seconds, found 2 clusters. [2026-01-21 04:27:49,430] INFO spindle_dev.index: Using epsilon-net clustering for block 31 [2026-01-21 04:27:49,460] INFO spindle_dev.index: Finished block 31 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:49,460] INFO spindle_dev.index: Using epsilon-net clustering for block 32 [2026-01-21 04:27:49,495] INFO spindle_dev.index: Finished block 32 in 0.03 seconds, found 3 clusters. [2026-01-21 04:27:49,496] INFO spindle_dev.index: Using epsilon-net clustering for block 33 [2026-01-21 04:27:49,526] INFO spindle_dev.index: Finished block 33 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:49,526] INFO spindle_dev.index: Using epsilon-net clustering for block 34 [2026-01-21 04:27:49,556] INFO spindle_dev.index: Finished block 34 in 0.03 seconds, found 2 clusters. [2026-01-21 04:27:49,557] INFO spindle_dev.index: Using epsilon-net clustering for block 35 [2026-01-21 04:27:49,736] INFO spindle_dev.index: Finished block 35 in 0.18 seconds, found 6 clusters. [2026-01-21 04:27:49,737] INFO spindle_dev.index: Using epsilon-net clustering for block 36 [2026-01-21 04:27:49,779] INFO spindle_dev.index: Finished block 36 in 0.04 seconds, found 2 clusters. [2026-01-21 04:27:49,780] INFO spindle_dev.index: Using epsilon-net clustering for block 37 [2026-01-21 04:27:49,817] INFO spindle_dev.index: Finished block 37 in 0.04 seconds, found 3 clusters. [2026-01-21 04:27:49,817] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:27:49,818] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:27:49,819] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:27:49,819] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:27:49,819] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:27:49,832] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:27:49,833] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances. [2026-01-21 04:27:49,834] INFO spindle_dev.index: Processing cluster 7 [2026-01-21 04:27:49,834] INFO spindle_dev.index: Building SPD index with epsilon=1.8774843590512265 [2026-01-21 04:27:49,835] INFO spindle_dev.index: Step 1: Cluster blocks within each class of SPD matrices. [2026-01-21 04:27:49,850] INFO spindle_dev.index: Cluster 7: 34 SPDs, 19 blocks [2026-01-21 04:27:49,851] INFO spindle_dev.index: Using epsilon-net clustering for block 0 [2026-01-21 04:27:49,855] INFO spindle_dev.index: Finished block 0 in 0.00 seconds, found 1 clusters. [2026-01-21 04:27:49,855] INFO spindle_dev.index: Using epsilon-net clustering for block 1 [2026-01-21 04:27:49,860] INFO spindle_dev.index: Finished block 1 in 0.00 seconds, found 6 clusters. [2026-01-21 04:27:49,861] INFO spindle_dev.index: Using epsilon-net clustering for block 2 [2026-01-21 04:27:49,866] INFO spindle_dev.index: Finished block 2 in 0.01 seconds, found 7 clusters. [2026-01-21 04:27:49,867] INFO spindle_dev.index: Using epsilon-net clustering for block 3 [2026-01-21 04:27:49,872] INFO spindle_dev.index: Finished block 3 in 0.01 seconds, found 8 clusters. [2026-01-21 04:27:49,873] INFO spindle_dev.index: Using epsilon-net clustering for block 4 [2026-01-21 04:27:49,879] INFO spindle_dev.index: Finished block 4 in 0.01 seconds, found 5 clusters. [2026-01-21 04:27:49,879] INFO spindle_dev.index: Using epsilon-net clustering for block 5 [2026-01-21 04:27:49,886] INFO spindle_dev.index: Finished block 5 in 0.01 seconds, found 9 clusters. [2026-01-21 04:27:49,886] INFO spindle_dev.index: Using epsilon-net clustering for block 6 [2026-01-21 04:27:49,896] INFO spindle_dev.index: Finished block 6 in 0.01 seconds, found 13 clusters. [2026-01-21 04:27:49,897] INFO spindle_dev.index: Using epsilon-net clustering for block 7 [2026-01-21 04:27:49,903] INFO spindle_dev.index: Finished block 7 in 0.01 seconds, found 8 clusters. [2026-01-21 04:27:49,903] INFO spindle_dev.index: Using epsilon-net clustering for block 8 [2026-01-21 04:27:49,910] INFO spindle_dev.index: Finished block 8 in 0.01 seconds, found 10 clusters. [2026-01-21 04:27:49,911] INFO spindle_dev.index: Using epsilon-net clustering for block 9 [2026-01-21 04:27:49,919] INFO spindle_dev.index: Finished block 9 in 0.01 seconds, found 14 clusters. [2026-01-21 04:27:49,920] INFO spindle_dev.index: Using epsilon-net clustering for block 10 [2026-01-21 04:27:49,928] INFO spindle_dev.index: Finished block 10 in 0.01 seconds, found 13 clusters. [2026-01-21 04:27:49,929] INFO spindle_dev.index: Using epsilon-net clustering for block 11 [2026-01-21 04:27:49,953] INFO spindle_dev.index: Finished block 11 in 0.02 seconds, found 33 clusters. [2026-01-21 04:27:49,954] INFO spindle_dev.index: Using epsilon-net clustering for block 12 [2026-01-21 04:27:49,960] INFO spindle_dev.index: Finished block 12 in 0.01 seconds, found 9 clusters. [2026-01-21 04:27:49,960] INFO spindle_dev.index: Using epsilon-net clustering for block 13 [2026-01-21 04:27:49,965] INFO spindle_dev.index: Finished block 13 in 0.00 seconds, found 5 clusters. [2026-01-21 04:27:49,966] INFO spindle_dev.index: Using epsilon-net clustering for block 14 [2026-01-21 04:27:49,972] INFO spindle_dev.index: Finished block 14 in 0.01 seconds, found 7 clusters. [2026-01-21 04:27:49,972] INFO spindle_dev.index: Using epsilon-net clustering for block 15 [2026-01-21 04:27:49,979] INFO spindle_dev.index: Finished block 15 in 0.01 seconds, found 11 clusters. [2026-01-21 04:27:49,980] INFO spindle_dev.index: Using epsilon-net clustering for block 16 [2026-01-21 04:27:49,986] INFO spindle_dev.index: Finished block 16 in 0.01 seconds, found 1 clusters. [2026-01-21 04:27:49,987] INFO spindle_dev.index: Using epsilon-net clustering for block 17 [2026-01-21 04:27:54,166] INFO spindle_dev.index: Finished block 17 in 4.18 seconds, found 1 clusters. [2026-01-21 04:27:54,171] INFO spindle_dev.index: Using epsilon-net clustering for block 18 [2026-01-21 04:27:58,882] INFO spindle_dev.index: Finished block 18 in 4.71 seconds, found 1 clusters. [2026-01-21 04:27:58,884] INFO spindle_dev.index: Step 2: Build DAG connections between block clusters. [2026-01-21 04:27:58,885] INFO spindle_dev.index: Step 2.1: For each layer order the block-clusters by [2026-01-21 04:27:58,886] INFO spindle_dev.index: Not implemented: ordering block-clusters ? How to order them? [2026-01-21 04:27:58,888] INFO spindle_dev.index: We will use triangle inequality to order clusters. [2026-01-21 04:27:58,889] INFO spindle_dev.index: Step 2.2: Connect block-clusters between layers based on co-occurrence in SPDs. [2026-01-21 04:27:58,892] INFO spindle_dev.index: Check if node global_node_id matches index in nodes list [2026-01-21 04:27:58,894] INFO spindle_dev.index: Step 2.3: Ordering block-clusters within each layer using log-Euclidean distances.
In [21]:
search_cfg = search.SearchConfig(max_results=2, debug=False, max_failed_starts=5, max_failed_paths=10)
In [22]:
test_results = test.run_sanity_search(data, dag_dict, config, search_cfg, max_queries=2000, skip_baseline=True)
[2026-01-21 04:29:05,337] INFO spindle_dev.test: Built ground-truth paths for 3053 SPDs in cluster 0 across 36 blocks. [2026-01-21 04:29:18,299] INFO spindle_dev.test: Built ground-truth paths for 2643 SPDs in cluster 1 across 36 blocks. [2026-01-21 04:29:22,302] INFO spindle_dev.test: Built ground-truth paths for 1523 SPDs in cluster 2 across 35 blocks. [2026-01-21 04:29:26,399] INFO spindle_dev.test: Built ground-truth paths for 1479 SPDs in cluster 3 across 38 blocks. [2026-01-21 04:29:28,941] INFO spindle_dev.test: Built ground-truth paths for 1196 SPDs in cluster 4 across 39 blocks. [2026-01-21 04:29:29,323] INFO spindle_dev.test: Built ground-truth paths for 460 SPDs in cluster 5 across 35 blocks. [2026-01-21 04:29:29,650] INFO spindle_dev.test: Built ground-truth paths for 414 SPDs in cluster 6 across 38 blocks. [2026-01-21 04:29:29,653] INFO spindle_dev.test: Built ground-truth paths for 34 SPDs in cluster 7 across 19 blocks. [2026-01-21 04:29:29,653] INFO spindle_dev.test: Created ground-truth paths for all clusters. [2026-01-21 04:29:29,654] INFO spindle_dev.test: Search config: SearchConfig(max_results=2, max_failed_starts=5, max_failed_paths=10, total_paths_limit=1000, deterministic=DeterministicConfig(seed=0), debug=False) [2026-01-21 04:29:29,659] INFO spindle_dev.test: Running sanity search with 2000 queries. 1%| | 24/2000 [00:07<10:27, 3.15it/s]
--------------------------------------------------------------------------- KeyboardInterrupt Traceback (most recent call last) Cell In[22], line 1 ----> 1 test_results = test.run_sanity_search(data, dag_dict, config, search_cfg, max_queries=2000, skip_baseline=True) File /data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/test.py:260, in run_sanity_search(data, dag_dict, config, search_cfg, max_queries, seed, skip_baseline) 258 logger.info("Created ground-truth paths for all clusters.") 259 logger.info("Search config: %s", search_cfg) --> 260 records = search_ground_truth_matrices( 261 data=data, 262 dag_dict=dag_dict, 263 config=config, 264 ground_truth_paths=gt_paths, 265 search_cfg=search_cfg, 266 max_queries=max_queries, 267 seed=seed, 268 skip_baseline=skip_baseline 269 ) 271 # Print a short summary to the log. 272 n = len(records) File /data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/test.py:183, in search_ground_truth_matrices(data, dag_dict, config, ground_truth_paths, search_cfg, max_queries, seed, skip_baseline) 181 if not search_cfg: 182 search_cfg = search.SearchConfig(max_results=2, debug=False, max_failed_starts=5, max_failed_paths=10) --> 183 results = search.search_index( 184 index_handle, 185 q_spd_perm, 186 [], 187 query_block_runs, 188 budget, 189 config=search_cfg, 190 ) 191 search_time = time.perf_counter() - t1 193 # Check if any returned path matches the full ground-truth path. File /data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/search.py:469, in search_index(index_handle, query_spd, query_indices, query_block_runs, budget, config) 462 logger.debug( 463 "Seed DFS from node_idx=%d dist=%.4f remaining_budget=%.4f", 464 node_idx, 465 dist, 466 remaining, 467 ) 468 before = len(best_paths) --> 469 dfs(0, node_idx, remaining, dist, [node_idx]) 471 # If this start contributed no new leaf paths, count it as a 472 # failed attempt. For hard / false-positive queries this 473 # provides a hard cap on search effort. 474 if len(best_paths) == before: File /data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/search.py:418, in search_index.<locals>.dfs(layer_idx, node_idx, remaining_budget, total_dist, path_indices) 408 if debug: 409 logger.debug( 410 "Recurse to child: layer=%d child_idx=%d dist=%.4f new_total=%.4f new_remaining=%.4f", 411 next_layer_idx, (...) 415 new_remaining, 416 ) --> 418 dfs(next_layer_idx, child_idx, new_remaining, new_total, path_indices + [child_idx]) 420 # Optional early stopping if we've already collected enough results. 421 if done: File /data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/search.py:418, in search_index.<locals>.dfs(layer_idx, node_idx, remaining_budget, total_dist, path_indices) 408 if debug: 409 logger.debug( 410 "Recurse to child: layer=%d child_idx=%d dist=%.4f new_total=%.4f new_remaining=%.4f", 411 next_layer_idx, (...) 415 new_remaining, 416 ) --> 418 dfs(next_layer_idx, child_idx, new_remaining, new_total, path_indices + [child_idx]) 420 # Optional early stopping if we've already collected enough results. 421 if done: [... skipping similar frames: search_index.<locals>.dfs at line 418 (26 times)] File /data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/search.py:418, in search_index.<locals>.dfs(layer_idx, node_idx, remaining_budget, total_dist, path_indices) 408 if debug: 409 logger.debug( 410 "Recurse to child: layer=%d child_idx=%d dist=%.4f new_total=%.4f new_remaining=%.4f", 411 next_layer_idx, (...) 415 new_remaining, 416 ) --> 418 dfs(next_layer_idx, child_idx, new_remaining, new_total, path_indices + [child_idx]) 420 # Optional early stopping if we've already collected enough results. 421 if done: File /data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/search.py:375, in search_index.<locals>.dfs(layer_idx, node_idx, remaining_budget, total_dist, path_indices) 373 L_block = log_spd(query_block_next) 374 diff = L_block - child.metadata.mean --> 375 dist = np.linalg.norm(diff, ord='fro') / np.sqrt(p) 376 child_dists.append((child_idx, dist)) 378 # Explore children from closest to farthest. File /panfs/accrepfs.vampire/home/sarkah1/miniforge3/envs/spatial/lib/python3.10/site-packages/numpy/linalg/_linalg.py:2738, in norm(x, ord, axis, keepdims) 2732 ndim = x.ndim 2733 if ( 2734 (ord is None) or 2735 (ord in ('f', 'fro') and ndim == 2) or 2736 (ord == 2 and ndim == 1) 2737 ): -> 2738 x = x.ravel(order='K') 2739 if isComplexType(x.dtype.type): 2740 x_real = x.real KeyboardInterrupt:
In [23]:
fig = plotting.visualize_block_dag_sankey_scaled(dag_dict[2].nodes, height=800, thickness=20.0)
fig
In [24]:
import pandas as pd
def get_dag_stats(dag_dict):
dag_stats = []
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
num_spds = len(node.metadata.members)
block_size = node.metadata.mean.shape[0]
block_id = node.block_index
radius = node.metadata.radius
#print(f"Cluster_id {cluster_id} Node id {node.global_node_id} Block {block_id}: {num_spds} spds")
dag_stats.append({
"cluster_id": cluster_id,
"node_id": node.global_node_id,
"block_id": block_id,
"block_cluster_id": node.block_cluster_id,
"num_spds": num_spds,
"radius": radius,
"block_size": block_size
})
dag_stats_df = pd.DataFrame(dag_stats)
return dag_stats_df
In [25]:
dag_stats_df = get_dag_stats(dag_dict)
In [26]:
score_list = []
for cluster_id in set(data.labels):
cluster_dag = dag_dict[cluster_id]
cluster_score = index.score_nodes_from_a_cluster(data, cluster_id, dag_dict, take_representative_mean=True)
score_list.extend(cluster_score)
100%|██████████| 341/341 [00:00<00:00, 412.31it/s] 100%|██████████| 131/131 [00:01<00:00, 124.72it/s] 100%|██████████| 113/113 [00:00<00:00, 171.14it/s] 100%|██████████| 136/136 [00:00<00:00, 160.77it/s] 100%|██████████| 234/234 [00:00<00:00, 412.48it/s] 100%|██████████| 232/232 [00:00<00:00, 270.62it/s] 100%|██████████| 117/117 [00:00<00:00, 129.68it/s] 100%|██████████| 162/162 [00:00<00:00, 218.94it/s]
In [27]:
node_scores = []
for node_score in score_list:
node_scores.append({
"cluster_id": node_score["cluster_id"],
"block_id": node_score["block_id"],
"node_id": node_score["node_id"],
"node_score": node_score["node_score"],
"num_spds": node_score["num_spds"],
"block_size": node_score["block_size"],
"E": node_score["E"],
"Q": node_score["Q"],
"S": node_score["S"],
"num_modules": len(node_score["modules"])
})
node_score_df = pd.DataFrame(node_scores)
In [28]:
node_score_df.sort_values(by='node_score', ascending=False).head(20)
Out[28]:
| cluster_id | block_id | node_id | node_score | num_spds | block_size | E | Q | S | num_modules | |
|---|---|---|---|---|---|---|---|---|---|---|
| 1296 | 6 | 35 | 109 | 0.358013 | 1 | 50 | 2.413795 | 0.246120 | 0.397371 | 4 |
| 1054 | 5 | 15 | 99 | 0.280287 | 14 | 15 | 1.063484 | 0.493565 | 0.466018 | 4 |
| 702 | 3 | 34 | 117 | 0.245047 | 1 | 49 | 2.530938 | 0.177834 | 0.455560 | 2 |
| 1297 | 6 | 35 | 110 | 0.230143 | 1 | 50 | 2.336212 | 0.171471 | 0.425498 | 4 |
| 1114 | 5 | 21 | 159 | 0.228424 | 4 | 40 | 1.072593 | 0.314746 | 0.323380 | 8 |
| 1294 | 6 | 35 | 107 | 0.223030 | 1 | 50 | 3.197471 | 0.181123 | 0.614894 | 4 |
| 424 | 1 | 25 | 83 | 0.220085 | 2 | 48 | 2.849021 | 0.169532 | 0.544341 | 4 |
| 1138 | 5 | 31 | 183 | 0.219983 | 1 | 48 | 2.301896 | 0.188482 | 0.492972 | 2 |
| 1299 | 6 | 36 | 112 | 0.219743 | 94 | 19 | 1.589406 | 0.277518 | 0.501819 | 3 |
| 453 | 1 | 35 | 112 | 0.218760 | 1 | 49 | 3.300771 | 0.164754 | 0.597734 | 2 |
| 305 | 0 | 23 | 305 | 0.213959 | 2 | 36 | 1.138367 | 0.366583 | 0.487285 | 4 |
| 455 | 1 | 35 | 114 | 0.210521 | 1 | 49 | 2.433158 | 0.162153 | 0.466420 | 3 |
| 692 | 3 | 34 | 107 | 0.208905 | 1 | 49 | 2.202111 | 0.164805 | 0.424377 | 2 |
| 696 | 3 | 34 | 111 | 0.206956 | 1 | 49 | 3.153194 | 0.155792 | 0.578713 | 3 |
| 575 | 2 | 32 | 103 | 0.205635 | 1 | 45 | 2.459674 | 0.172220 | 0.514565 | 4 |
| 707 | 3 | 34 | 122 | 0.201826 | 1 | 49 | 2.402435 | 0.178403 | 0.529109 | 2 |
| 449 | 1 | 35 | 108 | 0.200511 | 1 | 49 | 2.538474 | 0.159742 | 0.505526 | 3 |
| 694 | 3 | 34 | 109 | 0.199463 | 1 | 49 | 2.570456 | 0.164693 | 0.528834 | 3 |
| 68 | 0 | 8 | 68 | 0.196388 | 108 | 10 | 0.983023 | 0.460042 | 0.565736 | 5 |
| 708 | 3 | 34 | 123 | 0.192449 | 1 | 49 | 3.321746 | 0.124247 | 0.533707 | 3 |
In [33]:
importlib.reload(plotting)
Out[33]:
<module 'spindle_dev.plotting' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/plotting.py'>
In [29]:
#id = 513
id = 1296
scores_846, fig_846 = plotting.plot_module_heatmap_plus_spatial(
id,
score_list,
node_score_df,
dag_stats_df,
dag_dict,
data,
adata,
spot_size=0.2
)
Creating combined figure for cluster 6, node 109, block 35
In [ ]:
In [30]:
fig_846.savefig(f"{result_dir}/{id}_module_0_1_enrichment.png", dpi=300)
In [31]:
from spindle_dev import go_score
In [32]:
out_846 = go_score.enrich_modules_with_gseapy(score_list[id]['modules'])
Module 0: enrichment completed with 946 terms Module 1: enrichment completed with 670 terms Module 2: enrichment completed with 791 terms Module 3: enrichment completed with 166 terms
In [124]:
importlib.reload(go_score)
Out[124]:
<module 'spindle_dev.go_score' from '/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/go_score.py'>
In [33]:
for module_id, module_genes, res in out_846:
if module_id == 0:
print(module_genes)
fig, axes = go_score.plot_module_enrichment_libraries_2(
module_id=module_id,
module_genes=module_genes,
res=res,
top_n=7,
ncols=2,
bar_color="#D41515EE", # pick your color
tick_fontsize=10,
label_fontsize=10,
title_fontsize=8,
compact_height_per_term=0.2,
min_fig_h=8,
plot_kind="dot",
size_range=(20, 200),
use_constrained_layout=False,
save_dir=f"{result_dir}/go_enrichment/enrichment_{id}"
)
if fig is not None:
plt.show()
['ACTA2', 'ACTB', 'ACTG2', 'ANXA1', 'CAV1', 'CD276', 'CD38', 'CD44', 'CD47', 'HLA-B', 'IFNAR1', 'LAMP1', 'LAMP2', 'NOTCH2', 'PMP22', 'STAT3', 'VCAN', 'VEGFA']
/data/sarkar_lab/Projects/spindle_dev/src/spindle_dev/go_score.py:519: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
In [39]:
# Create a dictionary of tile_id to block_id
# Clusters do not share tile ids across them
from collections import defaultdict
tile_to_block_dict = {}
cluster_to_tile = defaultdict(set)
for cluster_id, cluster_dag in dag_dict.items():
block_to_node = cluster_dag.block_to_node_indices
for node in cluster_dag.nodes:
member_tile_ids = [tid for tid,_ in node.metadata.members]
for tile_id in member_tile_ids:
if not tile_id in tile_to_block_dict:
tile_to_block_dict[tile_id] = {}
tile_to_block_dict[tile_id][node.block_index] = node.block_cluster_id
cluster_to_tile[cluster_id].add(tile_id)
In [40]:
def get_spot_score_dict(block_id, cluster_id, tile_to_block_dict, grid=False):
spot_score = {}
tiles_of_cluster = cluster_to_tile[cluster_id]
for tid in tiles_of_cluster:
if tid not in tile_to_block_dict:
continue
c = tile_to_block_dict[tid][block_id]
t = data.metadata['tiles'][tid]
for id in t.idx:
spot_score[id] = c
fig, ax = plotting.plot_spot_module_scores(spot_score, tiles, adata, color='discrete', grid=grid)
return fig, ax
In [48]:
fig, ax = get_spot_score_dict(19, 4, tile_to_block_dict, grid=False)
In [ ]:
sc.pl.spatial(adata,color=['ARFGEF3'], spot_size=20, cmap='Reds')
/tmp/ipykernel_442280/2723282100.py:1: FutureWarning: Use `squidpy.pl.spatial_scatter` instead. sc.pl.spatial(adata,color=['ARFGEF3'], spot_size=10, cmap='Reds')
In [56]:
res.columns
Out[56]:
Index(['Gene_set', 'Term', 'Overlap', 'P-value', 'Adjusted P-value',
'Old P-value', 'Old Adjusted P-value', 'Odds Ratio', 'Combined Score',
'Genes'],
dtype='object')
In [ ]: